import os


def remove_duplicates(src_file, des_file=None):
    if not des_file:
        des_file = f'{src_file}.unique.txt'
    if os.path.exists(des_file):
        print(f'{des_file} already exists, please remove first')
        return
    print(f'reading {src_file}')
    with open(src_file, 'r', encoding='utf8', errors='ignore') as fr:
        with open(des_file, 'w', encoding='utf8') as fw:
            written_lines = set()
            for line in fr.readlines():
                if line in written_lines:
                    print(f'------removed {line}')
                    continue
                fw.write(line)
                written_lines.add(line)
    print(f'output uniques to {des_file}')


if __name__ == '__main__':
    # remove_duplicates(r'D:\dl\测试文件.txt',r'D:\dl\测试文件_unique.txt')
    remove_duplicates(r'D:\dl\测试文件.txt') #可省略输出文件，自动构造
